import re
import traceback
import base64
import datetime
from google.appengine.ext import db
from google.appengine.api import memcache

import logging
log = logging.getLogger("appdb")

import appparser
import apperr

class Site(db.Model):
  url = db.StringProperty()
  count = db.IntegerProperty(default = 0)
  
class Table(Site):
  date = db.DateTimeProperty(default = datetime.datetime.now())
  fmt = db.StringProperty(default = "%Y")
  unit = db.StringProperty()
  tags = db.TextProperty()

class Record(db.Model):
  date = db.DateTimeProperty()
  dfmt = db.StringProperty()
  value = db.FloatProperty()
  unit = db.StringProperty()
  source = db.StringProperty()

class Tag(db.Model):
  name = db.StringProperty()
  records = db.StringListProperty()
  rank = db.IntegerProperty(default = 0)
  related = db.TextProperty()
  
class TagIndex(db.Model):
  word = db.StringProperty()
  tags = db.TextProperty()
  
class RefreshTable(db.Model):
  value = db.StringProperty()
  url = db.StringProperty()
  
class History(db.Model):
  words = db.StringProperty()
  qcount = db.IntegerProperty(default = 1)
  found = db.BooleanProperty(default = True)
  timestamp = db.DateTimeProperty(default = datetime.datetime.now())
  
class ErrorReport(db.Model):
  url = db.StringProperty()
  count = db.IntegerProperty(default = 0)
  timestamp = db.DateTimeProperty(default = datetime.datetime.now())

class TableTBF(db.Model):
  url = db.StringProperty()
  reason = db.StringListProperty()

class Bookmark(db.Model):
  value = db.StringProperty()
  timestamp = db.DateTimeProperty(default = datetime.datetime.now())
  
class ScanSite(Site):
  def getScanned(self):
    return Site(key_name = GenURLKeyName("Site", self.url),
                url = self.url,
                count = self.count)

class ScanTable(Table):
  def getScanned(self):
    return Table(key_name = GenURLKeyName("Table", self.url),
                 url = self.url,
                 count = self.count,
                 date = self.date,
                 fmt = self.fmt,
                 unit = self.unit,
                 tags = self.tags)
    
class ScanRecord(Record):
  tags = db.TextProperty()
  def getScanned(self):
    return Record(key_name = GenRecordKeyName(self.date, unicode(self.tags).split("+")),
                  value = self.value,
                  unit = self.unit,
                  date = self.date,
                  dfmt = self.dfmt,
                  source = self.source)

class BatchCommand(object):
  def __init__(self, pagesize, batch=50):
    self.pagesize = pagesize
    self.batch = batch
    self.tobeAdd = {}
    self.tobeDel = {}
    
  def run(self):
    entities = self.entities()
    next = entities[-1] if entities else None
    success = True
    try:
      self.allEntities(entities)

      if len(self.tobeAdd): DBBatch(db.put, self.tobeAdd.values(), self.batch)
      if len(self.tobeDel): DBBatch(db.delete, self.tobeDel.values(), self.batch)
    except:
      log.exception(traceback.format_exc())
      success = False
    self.done(next, success)
    self.setBookmark(next)
    return len(entities)
  
  def allEntities(self, entities):
    for e in entities:
      if e is None: continue
      self.entity(e)
      self.scanned(e)

  def entities(self):
    bookmark = self.getBookmark()
    return GetPagedEntities(self.getEntityName(),
                          "" if bookmark is None else bookmark.value,
                          self.getCondition(),
                          self.pagesize)

  def getBookmark(self):
    return Bookmark.get_by_key_name(self.getPrefix() + self.getEntityName())
  
  def setBookmark(self, next):
    bookmark = Bookmark(key_name = self.getPrefix() + self.getEntityName())
    if next:
      bookmark.value = str(next.key())
      bookmark.put()
    else:
      bookmark.delete()

  def getCondition(self):
    return ""
  
  def getPrefix(self):
    return self.__class__.__name__
  
  def entity(self, e):
    return
  
  def scanned(self, e):
    self.tobeDel[self.getEntityKeyName(e)] = e
    new = e.getScanned()
    self.tobeAdd[self.getEntityKeyName(e)] = new

  def done(self, next, success):
    return
  
def GenRecordKeyName(date, tags):
  tags.sort()
  return 'r_'+base64.b64encode("+".join(tags).encode("utf-8")+date.strftime("%Y%m%d%H%M%S"))

def UnGenRecordKeyName(keyName):
  result = base64.b64decode(keyName[2:])
  return result[:len(result)-14].decode("utf-8").split("+"), result[len(result)-14:]

def GenTagKeyName(tag):
  return 't_'+base64.b64encode(tag.encode("utf-8"))

def GenHistoryKeyName(tags):
  tags.sort()
  return 'h_'+base64.b64encode("+".join(tags).encode("utf-8"))

def GenQueryRecordsByTagKey(tags):
  tags.sort()
  return 'q_'+base64.b64encode("+".join(tags).encode("utf-8"))

def GenTagBookmark(bookmark, direction, pagesize):
  if "next" == direction:
    if bookmark:
      return "tbk_"+direction+str(pagesize)+str(bookmark.encode("utf-8"))
    return "tbk_"+direction+str(pagesize)+base64.b64encode("FirstBookmark")    
  elif "related" == direction:
    bookmark.sort()
    return "tbkr_"+direction+str(pagesize)+"_".join(bookmark)

def GenTagIndexKeyName(word):
  return "ti_"+base64.b64encode(word.encode("utf-8"))

def GenURLKeyName(prefix, url):
  return "url_%s_%s" %(prefix, base64.b64encode(url))

def GenBookmarkKeyName(keyName):
  return "bm_"+base64.b64encode(keyName)

def GenQueryTagIndexByWords(words):
  return "qtiw_"+base64.b64encode(words.encode("utf-8"))

def GetRecord(date, tags):
  if len(GenRecordKeyName(date, tags)) <= 250:
    item = memcache.get(GenRecordKeyName(date, tags))
    if item is not None:
      return item
  ins = Record.get_by_key_name(GenRecordKeyName(date, tags))
  if ins is None:
    ins = ScanRecord.get_by_key_name(GenRecordKeyName(date, tags))
  if ins is not None \
  and len(GenRecordKeyName(date, tags)) <= 250:
    memcache.set(GenRecordKeyName(date, tags), ins, 3600)
    return ins
  return None

def GetTag(tag, fromDB = False):
  if not fromDB:
    item = memcache.get(GenTagKeyName(tag))
    if item is not None:
      return item
  ins = Tag.get_by_key_name(GenTagKeyName(tag))
  if ins is not None:
    memcache.set(GenTagKeyName(tag), ins, 3600)
    return ins
  return None

def GetHistory(found):
  if found:
    key = "history_words_found"
  else:
    key = "history_words_notfound"
  history = memcache.get(key)
  if history is None:
    history = {}
  return key, history    

def GetTagIndex(word, fromDB = False):
  if not fromDB:
    item = memcache.get(GenTagIndexKeyName(word))
    if item is not None:
      return item
  ins = TagIndex.get_by_key_name(GenTagIndexKeyName(word))
  if ins is not None:
    memcache.set(GenTagIndexKeyName(word), ins, 3600)
    return ins
  return None

def UpdateEntity(entities, exist):
  e = dict((k, (exist, v)) for k, v in entities.iteritems()) 
  memcache.set_multi(e, 3600)

def DBBatch(op, entities, batch=50):
  start = 0
  while len(entities[start:batch+start]) > 0:
    op(entities[start:batch+start])
    log.debug("DB %s %d entities OK." %(op, batch if len(entities) > batch else len(entities)))
    start += batch

def GetRecords(target, begin, end, bookmark, pagesize=100):
  """
  http://google-appengine.googlegroups.com/web/efficient_paging_using_key_instead_of_a_dedicated_unique_property.txt
  *Original starting query* 
  WHERE x > 0 AND x < 9
  *Bookmarkable starting query*
  WHERE x > 0 AND x < 9 ORDER BY x ASC, __key__ ASC
  *Derived queries for starting from bookmark entity B*
  WHERE x = B.x AND __key__ > B ORDER BY __key__ ASC
  WHERE x > B.x AND x < 9 ORDER BY x ASC, __key__ ASC 
  """

  gqls = ["SELECT %s FROM Record WHERE date>=:1 AND date<=:2 ORDER BY date ASC, __key__ ASC" %(target),
          "SELECT %s FROM Record WHERE date=:1 AND __key__>=:2 ORDER BY __key__ ASC" %(target),
          "SELECT %s FROM Record WHERE date>:1 AND date<=:2 ORDER BY date ASC, __key__ ASC" %(target),
          ]
  if bookmark is None:
    query = db.GqlQuery(gqls[0], begin, end)
    records = query.fetch(pagesize+1)
  else:
    query = db.GqlQuery(gqls[1], bookmark.date, bookmark.key())
    records = query.fetch(pagesize+1)
    query = db.GqlQuery(gqls[2], bookmark.date, end)
    records += query.fetch(pagesize+1 - len(records))
  if len(records)<pagesize+1:
    return records, None
  else:
    return records[:pagesize], Record.get(records[-1])
  
def QueryRecordsByTag(begin, end, tagNames):
  records = memcache.get(GenQueryRecordsByTagKey(tagNames))
  if records is None:
    tags = Tag.get_by_key_name([GenTagKeyName(tag) for tag in tagNames])
    if None in tags: raise apperr.NoData
    records = tags[0].records
    for tag in tags[1:]:
      records = set(records) & set(tag.records)
    if len(records):
      try:
        memcache.set(GenQueryRecordsByTagKey(tagNames), records, 3600)
      except:
        log.exception(traceback.format_exc())
  
  if len(records) > 1000 \
  and not begin \
  and not end:
    raise apperr.TooMuchRecord
  if not len(records): raise apperr.NoData
  
  if begin: matched, begin, format = appparser.IsDate(begin, datetime.datetime.now())      
  if end: matched, end, format = appparser.IsDate(end, datetime.datetime.now())
  if begin or end:
    if not begin: begin = datetime.datetime.strptime("1978", "%Y")
    if not end: end = datetime.datetime.now()
    bookmark = None
    pagesize = 999
    recKeys = []
    while True:
      results, bookmark = GetRecords("__key__", begin, end, bookmark, pagesize)
      recKeys += results
      if not bookmark: break
    records = set(records) & set([key.name() for key in recKeys])

  if len(records) > 1000: raise apperr.TooMuchRecord
  if not len(records): raise apperr.NoData
  
  records = set(Record.get_by_key_name(records))
  records.discard(None)
  if not len(records): raise apperr.NoData
  records = list(records)
  records.sort(cmp = lambda x,y: cmp(x.date, y.date)) 

  return records

def QueryTagIndexByWords(q):
  item = memcache.get(GenQueryTagIndexByWords(q))
  if item: return item
  
  if q.find("+") >= 0:
    words = q.split("+")
  else:
    words = q.split(" ")
  partialMatched = {}
  fullMatched = {}
  for word in words:
    QueryTagIndexByWord(word, fullMatched, partialMatched)
  partialMatched = partialMatched.items()
  partialMatched.sort(cmp=lambda x,y: cmp(y[1][0],x[1][0]))
  rank1st = GetMoreRelated(fullMatched, q).items()
  rank1st.sort(cmp=lambda x,y: cmp(y[1][0] - len(y[0]),x[1][0] - len(x[0])))
  fullMatched = fullMatched.items()
  fullMatched.sort(cmp=lambda x,y: cmp(y[1][0] - len(y[0]),x[1][0] - len(x[0])))
  memcache.set(GenQueryTagIndexByWords(q), rank1st + fullMatched + partialMatched, 3600)
  return rank1st + fullMatched + partialMatched

def QueryTagIndexByWord(word, fullMatched, partialMatched):
  indexes = set(TagIndex.get_by_key_name([GenTagIndexKeyName(i) for i in word]))
  indexes.discard(None)
  tags = []
  for index in indexes:
    tags += unicode(index.tags).split("+")
  for tag in set(tags):
    if re.match(u".*%s.*" %(word), tag, re.U):
      if fullMatched.has_key(tag):
        fullMatched[tag] = (fullMatched[tag][0] + len(word), fullMatched[tag][1] + list(set(word)))
      else:
        fullMatched[tag] = (len(word), list(set(word)))
    else:
      matched = list(set(tag) & set(word))
      if partialMatched.has_key(tag):
        partialMatched[tag] = (partialMatched[tag][0] + len(matched),
                               partialMatched[tag][1] + matched)
      else:
        partialMatched[tag] = (len(matched), matched)
        
def GetMoreRelated(fullMatched, q):
  tags = Tag.get_by_key_name([GenTagKeyName(tag) for tag in fullMatched.keys()])
  rank1st = {}
  for tag in tags:
    pub = list(set(unicode(tag.related).split("+")) & set(rank1st.keys() + fullMatched.keys()))
    if not len(pub): continue
    tmp = {}
    for i in pub:
      if rank1st.has_key(i): tmp[i] = rank1st[i]
      elif fullMatched.has_key(i): tmp[i] = fullMatched[i]
    pub = tmp.items()
    pub.sort(cmp=lambda x,y: cmp(y[1][0] - len(y[0]),x[1][0] - len(x[0])))
    pub = [i[0] for i in pub]
    rank1st[tag.name] = (fullMatched[tag.name][0], fullMatched[tag.name][1], pub)
    fullMatched.pop(tag.name)
  return rank1st

def GetNextTags(bookmark, prev, pagesize=10):
  """
  http://google-appengine.googlegroups.com/web/efficient_paging_using_key_instead_of_a_dedicated_unique_property.txt
  *Original starting query*
  ORDER BY x DESC
  *Bookmarkable starting query*
  ORDER BY x DESC, __key__ ASC
  *Derived queries for starting from bookmark entity B*
  WHERE x = B.x AND __key__ > B ORDER BY __key__ ASC
  WHERE x < B.x ORDER BY x DESC, __key__ ASC
  """
  paging = memcache.get(GenTagBookmark(bookmark, "next", pagesize))
  if paging is not None:
    tags = paging[0]
    if len(tags) \
    and len(tags) == pagesize:
      return tags, tags[-1], paging[1]
    else:
      return tags, None, paging[1]
      
  gqls = ["ORDER BY rank DESC, __key__ ASC",
          "WHERE rank = :1 AND __key__>:2 ORDER BY __key__ ASC",
          "WHERE rank < :1 ORDER BY rank DESC, __key__ ASC",
          ]
  tag = GetTag(bookmark)
  if tag is None:
    query = Tag.gql(gqls[0])
    results = query.fetch(pagesize)
  else:
    query1 = Tag.gql(gqls[1], tag.rank, tag.key())
    query2 = Tag.gql(gqls[2], tag.rank)
    results = query1.fetch(pagesize)
    results += query2.fetch(pagesize-len(results))
  tags = [tag.name for tag in results]
  memcache.set(GenTagBookmark(bookmark, "next", pagesize), (tags, prev), 3600)
  if len(tags)<pagesize:
    return tags, None, prev
  else:
    return tags, tags[-1], prev
    
def GetRelatedTags(bookmark, pagesize=10):
  tagNames = bookmark.split(" ")
  tagNames.sort()
  related = memcache.get(GenTagBookmark(tagNames, "related", pagesize))
  if related is not None:
    if len(related):
      return related, related[-1]
    else:
      return related, None
  
  tags = Tag.get_by_key_name([GenTagKeyName(tag) for tag in tagNames])
  if None in tags: raise apperr.NullTag, "Part of Tags[%s] not found." %(bookmark)
  related = []
  for tag in tags:
    if not len(tag.related):
      insrelated = []
    else:
      insrelated = unicode(tag.related).split("+")
    if not len(related):
      related = insrelated
    else:
      related = set(related) & set(insrelated)
  related = list(related)
  memcache.set(GenTagBookmark(tagNames, "related", pagesize), related, 3600)
  return related, None 

def UpdateHistory(words, found = True):
  key, history = GetHistory(found)
  if len(history) >= 50:
    history = FlushHistory(history, found)
    
  if history.has_key(words):
    history[words] = (history[words][0]+1, datetime.datetime.now())
  else:
    history[words] = (1, datetime.datetime.now())
  memcache.set(key, history)

def FlushHistory(history, found):
  history = history.items()
  history.sort(cmp = lambda x,y: cmp(y[1][0], x[1][0]))
  i = 0
  histories = []
  while i < 20 \
  and i < len(history):
    if history[i][0]:
      histories.append(History(key_name = "h"+str(found)+str(i),
                               words = history[i][0],
                               qcount = history[i][1][0],
                               found = found,
                               timestamp = history[i][1][1]))
    i += 1
  DBBatch(db.put, histories)
  return {}
    
def GetSite(url):
  return ScanSite.get_by_key_name(GenURLKeyName("Site", url))

def GetTable(url):
  return ScanTable.get_by_key_name(GenURLKeyName("Table", url))

def GetPagedEntities(model, bookmark, condition, pagesize=10):
  """
  http://google-appengine.googlegroups.com/web/efficient_paging_using_key_instead_of_a_dedicated_unique_property.txt
  *Original starting query*
  WHERE x = 0
  *Bookmarkable starting query*
  WHERE x = 0 ORDER BY __key__ ASC
  *Derived queries for starting from bookmark entity B*
  WHERE x = 0 AND __key__ > B ORDER BY __key__ ASC
  """
  if condition:
    startCond = "WHERE %s" %(condition)
    derivedCond = "%s AND" %(condition)
  else:
    startCond = ""
    derivedCond = ""
    
  gqls = ["SELECT * FROM %s %s ORDER BY __key__ ASC" %(model, startCond),
          "SELECT * FROM %s WHERE %s __key__>:1 ORDER BY __key__ ASC" %(model, derivedCond),
          ]
  if not bookmark:
    query = db.GqlQuery(gqls[0])
    results = query.fetch(pagesize)
  else:
    query = db.GqlQuery(gqls[1], db.Key(bookmark))
    results = query.fetch(pagesize)
  return results

def GetErrorReport(url):
  return ErrorReport.get_by_key_name(GenURLKeyName("ErrorReport", url))

def GetRank(tag):
  return len(tag.records)

def DelRecordFromTag(record, tag, tobeAdd, tobeDel):
  tag.records.remove(record)
  tag.rank = GetRank(tag)
  if not len(tag.records):
    tobeDel[tag.key().name()] = tag
    DelTagFromRelated(tag, tobeAdd, tobeDel)
    DelTagFromIndex(tag, tobeAdd, tobeDel)
  else:
    tobeAdd[tag.key().name()] = tag

def DelTagFromRelated(tag, tobeAdd, tobeDel):
  for r in unicode(tag.related).split("+"):
    r = ScratchTag(r, tobeAdd, tobeDel)
    if r is None: continue
    related = unicode(r.related).split("+")
    if tag.name not in related: continue
    related.remove(tag.name)
    r.related = "+".join(related)
    tobeAdd[r.key().name()] = r  
  
def DelTagFromIndex(tag, tobeAdd, tobeDel):
  indexes = ScratchTagIndexes(tag.name, tobeAdd, tobeDel)
  for tagIndex in indexes:
    related = unicode(tagIndex.tags).split("+")
    if tag.name not in related: continue
    related.remove(tag.name)
    if not len(related):
      tobeDel[tagIndex.key().name()] = tagIndex
    else:
      tagIndex.tags = "+".join(related)
      tobeAdd[tagIndex.key().name()] = tagIndex

def ScratchTag(tag, tobeAdd, tobeDel):
  if tobeDel.has_key(GenTagKeyName(tag)):
    return None
  elif tobeAdd.has_key(GenTagKeyName(tag)):
    return tobeAdd[GenTagKeyName(tag)]
  else:
    return Tag.get_by_key_name(GenTagKeyName(tag))  

def ScratchTags(tags, tobeAdd, tobeDel):
  fromDict = []
  fromDB = []
  for tag in tags:
    if tobeDel.has_key(GenTagKeyName(tag)):
      continue
    elif tobeAdd.has_key(GenTagKeyName(tag)):
      fromDict.append(tobeAdd[GenTagKeyName(tag)])
    else:
      fromDB.append(tag)
  return fromDict + Tag.get_by_key_name([GenTagKeyName(tag) for tag in fromDB])  

def ScratchTagIndexes(indexes, tobeAdd, tobeDel):
  fromDict = []
  fromDB = []
  for word in indexes:
    if tobeDel.has_key(GenTagIndexKeyName(word)):
      continue
    elif tobeAdd.has_key(GenTagIndexKeyName(word)):
      fromDict.append(tobeAdd[GenTagIndexKeyName(word)])
    else:
      fromDB.append(word)
  return fromDict + TagIndex.get_by_key_name([GenTagIndexKeyName(word) for word in fromDB])
